{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据老师所给代码进行运行修改参数和练习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from requests_html import HTMLSession\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "edu         5\n",
      "经验         10\n",
      "薪水         67\n",
      "时间         24\n",
      "职称        161\n",
      "公司地点       70\n",
      "公司名称       70\n",
      "链结        171\n",
      "公司URL      70\n",
      "热门公司类型      5\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>职称</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>公司名称</th>\n",
       "      <th>edu</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>明略科技集团</th>\n",
       "      <th>统招本科</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>小米</th>\n",
       "      <th>统招本科</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">华为</th>\n",
       "      <th>本科及以上</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>统招本科</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>深圳市优必选科技股份有限公司</th>\n",
       "      <th>本科及以上</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>中源家居</th>\n",
       "      <th>统招本科</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">欧菲光</th>\n",
       "      <th>大专及以上</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>本科及以上</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>安徽广信农化股份有限公司</th>\n",
       "      <th>统招本科</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>平安银行</th>\n",
       "      <th>本科及以上</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      职称\n",
       "公司名称           edu      \n",
       "明略科技集团         统招本科   12\n",
       "小米             统招本科   12\n",
       "华为             本科及以上  12\n",
       "               统招本科    8\n",
       "深圳市优必选科技股份有限公司 本科及以上   6\n",
       "...                   ..\n",
       "中源家居           统招本科    1\n",
       "欧菲光            大专及以上   1\n",
       "               本科及以上   1\n",
       "安徽广信农化股份有限公司   统招本科    1\n",
       "平安银行           本科及以上   1\n",
       "\n",
       "[93 rows x 1 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 按照老师所给代码进行参数更改尝试练习，将“中国500强”变为“2018互联网300强”\n",
    "参数_compTag_用户体验 = {'2018互联网300强': {'init': ['-1'], 'headckid': ['cdf63d8fd1ef8e7f'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'keyword': ['用户体验'], 'compTag': ['182'], 'ckid': ['a4bdf7ce86fc28d1'], 'siTag': ['8iR9k8w0PtD9mZEhJ_u3lQ~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_unknown'], 'd_ckId': ['6aa779111c1b4ca77cff3648d9dee049'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['6aa779111c1b4ca77cff3648d9dee049']}, '2018互联网300强': {'init': ['-1'], 'headckid': ['58d828c357a8cb19'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'keyword': ['用户体验'], 'compTag': ['182'], 'ckid': ['58d828c357a8cb19'], 'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_unknown'], 'd_ckId': ['6aa779111c1b4ca77cff3648d9dee049'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['6aa779111c1b4ca77cff3648d9dee049']}, '制造业500强': {'init': ['-1'], 'headckid': ['58d828c357a8cb19'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'keyword': ['用户体验'], 'compTag': ['186'], 'ckid': ['58d828c357a8cb19'], 'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_unknown'], 'd_ckId': ['6aa779111c1b4ca77cff3648d9dee049'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['6aa779111c1b4ca77cff3648d9dee049']}, 'AI创新成长50强 ': {'init': ['-1'], 'headckid': ['58d828c357a8cb19'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'keyword': ['用户体验'], 'compTag': ['189'], 'ckid': ['58d828c357a8cb19'], 'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_unknown'], 'd_ckId': ['6aa779111c1b4ca77cff3648d9dee049'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['6aa779111c1b4ca77cff3648d9dee049']}, '独角兽': {'init': ['-1'], 'headckid': ['58d828c357a8cb19'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'keyword': ['用户体验'], 'compTag': ['130'], 'ckid': ['58d828c357a8cb19'], 'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_unknown'], 'd_ckId': ['6aa779111c1b4ca77cff3648d9dee049'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['6aa779111c1b4ca77cff3648d9dee049']}, '上市公司': {'init': ['-1'], 'headckid': ['58d828c357a8cb19'], 'flushckid': ['1'], 'fromSearchBtn': ['2'], 'keyword': ['用户体验'], 'compTag': ['156'], 'ckid': ['58d828c357a8cb19'], 'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_unknown'], 'd_ckId': ['6aa779111c1b4ca77cff3648d9dee049'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['6aa779111c1b4ca77cff3648d9dee049']}}\n",
    "\n",
    "# 上周C-1   多个页面准备测试1 2018互联网300强\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "session = HTMLSession()\n",
    "payload = 参数_compTag_用户体验['2018互联网300强']\n",
    "r = session.get( url, params = payload)\n",
    "\n",
    "# r.url\n",
    "\n",
    "# 上周C-2  简化 A-1   单一页面爬+解析\n",
    "session = HTMLSession()\n",
    "\n",
    "def requests_liepin( url, params):\n",
    "    r = session.get( url , params = payload)\n",
    "\n",
    "    # 先取特定元素, 精准打击其子后辈\n",
    "    主要元素 = r.html.xpath( '//ul[@class=\"sojob-list\"]/li')\n",
    "\n",
    "    # 作为xpath字典，键为我要抓的牛肉名称，值为xpath\n",
    "    dict_xpaths={ \n",
    "        'text': {\n",
    "            'edu':      '//div[contains(@class,\"job-info\")]/p/span[@class=\"edu\"]',\n",
    "            '经验':      '//div[contains(@class,\"job-info\")]/p/span[@class=\"edu\"]/following-sibling::span',\n",
    "            '薪水':    '//div[contains(@class,\"job-info\")]/p/span[@class=\"text-warning\"]', \n",
    "            '时间':    '//div[contains(@class,\"job-info\")]/p/time/@title', \n",
    "            '职称':    '//div[contains(@class,\"job-info\")]/h3/a', \n",
    "            '公司地点': '//div[contains(@class,\"job-info\")]/p/a',\n",
    "            '公司名称': '//div[contains(@class,\"sojob-item-main\")]//p[@class=\"company-name\"]/a', \n",
    "        },\n",
    "        'text_content': {\n",
    "        },\n",
    "        'href': {\n",
    "            '链结':    '//div[contains(@class,\"job-info\")]/h3/a', \n",
    "            '公司URL': '//div[contains(@class,\"sojob-item-main\")]//p[@class=\"company-name\"]/a', \n",
    "        }\n",
    "    }\n",
    "\n",
    "    def get_e_text_content(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [\"\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_href(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [list(e.xpath(_xpath_, first=True).absolute_links)[0] \\\n",
    "                   if len(e.xpath(_xpath_, first=True).absolute_links) >= 1  \\\n",
    "                   else \"\" for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    # 只对主要元素下进行.xpath取值\n",
    "    数据字典 = dict()\n",
    "\n",
    "    数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "    数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "\n",
    "    数据 = pd.DataFrame(数据字典)\n",
    "    #数据.to_excel(\"20春_Web数据挖掘_week03_liepin.xlsx\", sheet_name=\"搜查结果\")\n",
    "    return (数据)\n",
    "\n",
    "\n",
    "# 上周C-3   多个页面\n",
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "\n",
    "list_df = list()\n",
    "for k,v in 参数_compTag_用户体验.items():\n",
    "    payload = v\n",
    "    df = requests_liepin( url, params = payload)\n",
    "    df = df.assign (热门公司类型 = k)    \n",
    "    list_df.append(df)\n",
    "\n",
    "df_all = pd.concat(list_df)\n",
    "df_all\n",
    "\n",
    "# 上周C-4   输出\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week03_liepin_各热门公司类型.xlsx\", sheet_name=\"搜查结果\")\n",
    "\n",
    "# 上周C-5 Pandas  基本能力\n",
    "\n",
    "print (df_all.nunique())\n",
    "df_all[['edu']].drop_duplicates()\n",
    "\n",
    "df_all.groupby(['公司名称','edu']).agg({\"职称\":\"count\"}).sort_values(by='职称', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 关键词为用户体验，公司选择为：2018互联网300强\n",
    "/zhaopin/?isAnalysis=&dqs=&pubTime=&salary=&subIndustry=&industryType=&compscale=&key=用户体验&init=-1&searchType=1&headckid=a4bdf7ce86fc28d1&flushckid=1&compkind=&fromSearchBtn=2&compTag=182&sortFlag=15&ckid=a4bdf7ce86fc28d1&jobKind=&industries=&clean_condition=&siTag=8iR9k8w0PtD9mZEhJ_u3lQ~fA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_prime&d_ckId=0e56fbfca6c1e42b91f3295791dd313f&d_curPage=0&d_pageSize=40&d_headId=0e56fbfca6c1e42b91f3295791dd313f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 翻页：参数字典的拆解 \n",
    "# 修改url\n",
    "url = \"https://www.liepin.com/zhaopin/?keyword=UX\"\n",
    "session = HTMLSession()\n",
    "r = session.get( url )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element 'a' href='/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=1'>, <Element 'a' href='/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=2'>, <Element 'a' href='/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=3'>, <Element 'a' href='/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=4'>, <Element 'a' href='/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=1'>, <Element 'a' class=('last',) href='/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=9' title='末页'>]\n",
      "{'2': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=1', '3': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=2', '4': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=3', '5': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=4', '下一页': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=1', '': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=9'}\n"
     ]
    }
   ],
   "source": [
    "# A-1  xpath 解析翻页a/@href\n",
    "xpath_翻页a = '//div[@class=\"pagerbar\"]/a' # 有disabled, current等href是javascript\n",
    "xpath_翻页a = '//div[@class=\"pagerbar\"]/a[starts-with(@href,\"/zhaopin\")]'\n",
    "print (r.html.xpath(xpath_翻页a)) # 物件\n",
    "\n",
    "href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "#print (href_列表)\n",
    "\n",
    "文字_列表 = [x.text for x in r.html.xpath(xpath_翻页a)]\n",
    "#print (文字_列表)\n",
    "\n",
    "href_字典 = {x.text:x.xpath('//@href')[0]  for x in r.html.xpath(xpath_翻页a)}\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scheme</th>\n",
       "      <th>netloc</th>\n",
       "      <th>path</th>\n",
       "      <th>params</th>\n",
       "      <th>query</th>\n",
       "      <th>fragment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>init=-1&amp;headckid=49c56ce3e881c9f8&amp;fromSearchBt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>init=-1&amp;headckid=49c56ce3e881c9f8&amp;fromSearchBt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>init=-1&amp;headckid=49c56ce3e881c9f8&amp;fromSearchBt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>init=-1&amp;headckid=49c56ce3e881c9f8&amp;fromSearchBt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>init=-1&amp;headckid=49c56ce3e881c9f8&amp;fromSearchBt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>/zhaopin/</td>\n",
       "      <td></td>\n",
       "      <td>init=-1&amp;headckid=49c56ce3e881c9f8&amp;fromSearchBt...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  scheme netloc       path params  \\\n",
       "0                /zhaopin/          \n",
       "1                /zhaopin/          \n",
       "2                /zhaopin/          \n",
       "3                /zhaopin/          \n",
       "4                /zhaopin/          \n",
       "5                /zhaopin/          \n",
       "\n",
       "                                               query fragment  \n",
       "0  init=-1&headckid=49c56ce3e881c9f8&fromSearchBt...           \n",
       "1  init=-1&headckid=49c56ce3e881c9f8&fromSearchBt...           \n",
       "2  init=-1&headckid=49c56ce3e881c9f8&fromSearchBt...           \n",
       "3  init=-1&headckid=49c56ce3e881c9f8&fromSearchBt...           \n",
       "4  init=-1&headckid=49c56ce3e881c9f8&fromSearchBt...           \n",
       "5  init=-1&headckid=49c56ce3e881c9f8&fromSearchBt...           "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scheme      1\n",
      "netloc      1\n",
      "path        1\n",
      "params      1\n",
      "query       5\n",
      "fragment    1\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>init</th>\n",
       "      <th>headckid</th>\n",
       "      <th>fromSearchBtn</th>\n",
       "      <th>keyword</th>\n",
       "      <th>ckid</th>\n",
       "      <th>siTag</th>\n",
       "      <th>d_sfrom</th>\n",
       "      <th>d_ckId</th>\n",
       "      <th>d_curPage</th>\n",
       "      <th>d_pageSize</th>\n",
       "      <th>d_headId</th>\n",
       "      <th>curPage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>49c56ce3e881c9f8</td>\n",
       "      <td>2</td>\n",
       "      <td>UX</td>\n",
       "      <td>49c56ce3e881c9f8°radeFlag=0</td>\n",
       "      <td>1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw</td>\n",
       "      <td>search_unknown</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1</td>\n",
       "      <td>49c56ce3e881c9f8</td>\n",
       "      <td>2</td>\n",
       "      <td>UX</td>\n",
       "      <td>49c56ce3e881c9f8°radeFlag=0</td>\n",
       "      <td>1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw</td>\n",
       "      <td>search_unknown</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1</td>\n",
       "      <td>49c56ce3e881c9f8</td>\n",
       "      <td>2</td>\n",
       "      <td>UX</td>\n",
       "      <td>49c56ce3e881c9f8°radeFlag=0</td>\n",
       "      <td>1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw</td>\n",
       "      <td>search_unknown</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1</td>\n",
       "      <td>49c56ce3e881c9f8</td>\n",
       "      <td>2</td>\n",
       "      <td>UX</td>\n",
       "      <td>49c56ce3e881c9f8°radeFlag=0</td>\n",
       "      <td>1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw</td>\n",
       "      <td>search_unknown</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1</td>\n",
       "      <td>49c56ce3e881c9f8</td>\n",
       "      <td>2</td>\n",
       "      <td>UX</td>\n",
       "      <td>49c56ce3e881c9f8°radeFlag=0</td>\n",
       "      <td>1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw</td>\n",
       "      <td>search_unknown</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1</td>\n",
       "      <td>49c56ce3e881c9f8</td>\n",
       "      <td>2</td>\n",
       "      <td>UX</td>\n",
       "      <td>49c56ce3e881c9f8°radeFlag=0</td>\n",
       "      <td>1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw</td>\n",
       "      <td>search_unknown</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>90013c3c9e43180a8f29b22a2c20ed57</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  init          headckid fromSearchBtn keyword                         ckid  \\\n",
       "0   -1  49c56ce3e881c9f8             2      UX  49c56ce3e881c9f8°radeFlag=0   \n",
       "1   -1  49c56ce3e881c9f8             2      UX  49c56ce3e881c9f8°radeFlag=0   \n",
       "2   -1  49c56ce3e881c9f8             2      UX  49c56ce3e881c9f8°radeFlag=0   \n",
       "3   -1  49c56ce3e881c9f8             2      UX  49c56ce3e881c9f8°radeFlag=0   \n",
       "4   -1  49c56ce3e881c9f8             2      UX  49c56ce3e881c9f8°radeFlag=0   \n",
       "5   -1  49c56ce3e881c9f8             2      UX  49c56ce3e881c9f8°radeFlag=0   \n",
       "\n",
       "                                           siTag         d_sfrom  \\\n",
       "0  1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw  search_unknown   \n",
       "1  1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw  search_unknown   \n",
       "2  1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw  search_unknown   \n",
       "3  1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw  search_unknown   \n",
       "4  1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw  search_unknown   \n",
       "5  1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw  search_unknown   \n",
       "\n",
       "                             d_ckId d_curPage d_pageSize  \\\n",
       "0  90013c3c9e43180a8f29b22a2c20ed57         0         40   \n",
       "1  90013c3c9e43180a8f29b22a2c20ed57         0         40   \n",
       "2  90013c3c9e43180a8f29b22a2c20ed57         0         40   \n",
       "3  90013c3c9e43180a8f29b22a2c20ed57         0         40   \n",
       "4  90013c3c9e43180a8f29b22a2c20ed57         0         40   \n",
       "5  90013c3c9e43180a8f29b22a2c20ed57         0         40   \n",
       "\n",
       "                           d_headId curPage  \n",
       "0  90013c3c9e43180a8f29b22a2c20ed57       1  \n",
       "1  90013c3c9e43180a8f29b22a2c20ed57       2  \n",
       "2  90013c3c9e43180a8f29b22a2c20ed57       3  \n",
       "3  90013c3c9e43180a8f29b22a2c20ed57       4  \n",
       "4  90013c3c9e43180a8f29b22a2c20ed57       1  \n",
       "5  90013c3c9e43180a8f29b22a2c20ed57       9  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "init             1\n",
      "headckid         1\n",
      "fromSearchBtn    1\n",
      "keyword          1\n",
      "ckid             1\n",
      "siTag            1\n",
      "d_sfrom          1\n",
      "d_ckId           1\n",
      "d_curPage        1\n",
      "d_pageSize       1\n",
      "d_headId         1\n",
      "curPage          5\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "from urllib.parse import urlparse, parse_qs\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# 总体目标：输入 href_列表, 建构出参数字典\n",
    "\n",
    "# urlparse 解析后丢入数据框\n",
    "df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "\n",
    "display(df)\n",
    "print(df.nunique())\n",
    "display(df_qs)\n",
    "print(df_qs.nunique())\n",
    "\n",
    "df_qs.curPage\n",
    "df_qs = df_qs.assign (curPage_int=df_qs.curPage.astype(int)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'init': ['-1'], 'headckid': ['49c56ce3e881c9f8'], 'fromSearchBtn': ['2'], 'keyword': ['UX'], 'ckid': ['49c56ce3e881c9f8°radeFlag=0'], 'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'], 'd_sfrom': ['search_unknown'], 'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'], 'd_curPage': ['0'], 'd_pageSize': ['40'], 'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'], 'curPage': ['1']}\n",
      "{'2': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=1', '3': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=2', '4': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=3', '5': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=4', '下一页': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=1', '': '/zhaopin/?init=-1&headckid=49c56ce3e881c9f8&fromSearchBtn=2&keyword=UX&ckid=49c56ce3e881c9f8°radeFlag=0&siTag=1B2M2Y8AsgTpgAmY7PhCfg%7EfA9rXquZc5IkJpXC-Ycixw&d_sfrom=search_unknown&d_ckId=90013c3c9e43180a8f29b22a2c20ed57&d_curPage=0&d_pageSize=40&d_headId=90013c3c9e43180a8f29b22a2c20ed57&curPage=9'}\n"
     ]
    }
   ],
   "source": [
    "def parse_url_qs_for_curPage (url):\n",
    "    six_parts = urlparse(url) \n",
    "    out = parse_qs(six_parts.query)\n",
    "    return (out)\n",
    "\n",
    "# 取一例做模板\n",
    "参数模板 = parse_url_qs_for_curPage(href_列表[0])\n",
    "print (参数模板)\n",
    "\n",
    "print (href_字典)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "9\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [0]},\n",
       " 1: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [1]},\n",
       " 2: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [2]},\n",
       " 3: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [3]},\n",
       " 4: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [4]},\n",
       " 5: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [5]},\n",
       " 6: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [6]},\n",
       " 7: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [7]},\n",
       " 8: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [8]},\n",
       " 9: {'init': ['-1'],\n",
       "  'headckid': ['49c56ce3e881c9f8'],\n",
       "  'fromSearchBtn': ['2'],\n",
       "  'keyword': ['用户体验'],\n",
       "  'ckid': ['49c56ce3e881c9f8°radeFlag=0'],\n",
       "  'siTag': ['1B2M2Y8AsgTpgAmY7PhCfg~fA9rXquZc5IkJpXC-Ycixw'],\n",
       "  'd_sfrom': ['search_unknown'],\n",
       "  'd_ckId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'd_curPage': ['0'],\n",
       "  'd_pageSize': ['40'],\n",
       "  'd_headId': ['90013c3c9e43180a8f29b22a2c20ed57'],\n",
       "  'curPage': [9]}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def 参数模板生成(keyword, curPage):\n",
    "    参数 = 参数模板.copy()\n",
    "    参数['curPage'] = curPage\n",
    "    参数['keyword'] = keyword\n",
    "    return (参数)\n",
    "\n",
    "参数_keyword_用户体验_curPage = { \n",
    "    i:参数模板生成(curPage = [i], \\\n",
    "                  keyword = ['用户体验']) \\\n",
    "    for i,v in href_字典.items()\\\n",
    "    }\n",
    "\n",
    "# print(参数_keyword_用户体验_curPage) # 只生成本页有的额外翻页URL, 并没有推估到&curPage=9,也没有这页\n",
    "\n",
    "print (df_qs.curPage_int.min()) # 最小值只有1\n",
    "print (df_qs.curPage_int.max()) # 最大值只有9\n",
    "\n",
    "# 应该是 0 (本页)....9(最大值)\n",
    "\n",
    "参数_keyword_用户体验_curPage = { \n",
    "    i:参数模板生成(curPage = [i], \\\n",
    "                  keyword = ['用户体验']) \\\n",
    "    for i in range(0,df_qs.curPage_int.max()+1)\\\n",
    "    }\n",
    "参数_keyword_用户体验_curPage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = HTMLSession()\n",
    "\n",
    "def requests_liepin( url, params):\n",
    "    r = session.get( url , params = payload)\n",
    "\n",
    "    # 先取特定元素, 精准打击其子后辈\n",
    "    主要元素 = r.html.xpath( '//ul[@class=\"sojob-list\"]/li')\n",
    "\n",
    "    # 作为xpath字典，键为我要抓的牛肉名称，值为xpath\n",
    "    dict_xpaths={ \n",
    "        'text': {\n",
    "            'edu':      '//div[contains(@class,\"job-info\")]/p/span[@class=\"edu\"]',\n",
    "            '经验':      '//div[contains(@class,\"job-info\")]/p/span[@class=\"edu\"]/following-sibling::span',\n",
    "            '薪水':    '//div[contains(@class,\"job-info\")]/p/span[@class=\"text-warning\"]', \n",
    "            '时间':    '//div[contains(@class,\"job-info\")]/p/time/@title', \n",
    "            '职称':    '//div[contains(@class,\"job-info\")]/h3/a', \n",
    "            '公司地点': '//div[contains(@class,\"job-info\")]/p/a',\n",
    "            '公司名称': '//div[contains(@class,\"sojob-item-main\")]//p[@class=\"company-name\"]/a', \n",
    "        },\n",
    "        'text_content': {\n",
    "        },\n",
    "        'href': {\n",
    "            '链结':    '//div[contains(@class,\"job-info\")]/h3/a', \n",
    "            '公司URL': '//div[contains(@class,\"sojob-item-main\")]//p[@class=\"company-name\"]/a', \n",
    "        }\n",
    "    }\n",
    "\n",
    "    def get_e_text_content(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [e.xpath(_xpath_)[0].lxml.text_content() for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_text(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [\"\".join([x.strip() if type(x) is str else x.text.strip() for x in e.xpath(_xpath_)]) for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    def get_e_href(_xpath_):\n",
    "        # 高级列表推导\n",
    "        暂存结果 = [list(e.xpath(_xpath_, first=True).absolute_links)[0] \\\n",
    "                   if len(e.xpath(_xpath_, first=True).absolute_links) >= 1  \\\n",
    "                   else \"\" for e in 主要元素]\n",
    "        return(暂存结果)\n",
    "\n",
    "    # 只对主要元素下进行.xpath取值\n",
    "    数据字典 = dict()\n",
    "\n",
    "    数据字典 = {k:get_e_text_content(v) for k,v in dict_xpaths['text_content'].items()}\n",
    "    数据字典.update({k:get_e_text(v) for k,v in dict_xpaths['text'].items()})\n",
    "    数据字典.update({k:get_e_href(v) for k,v in dict_xpaths['href'].items()})\n",
    "\n",
    "    数据 = pd.DataFrame(数据字典)\n",
    "    #数据.to_excel(\"20春_Web数据挖掘_week03_liepin.xlsx\", sheet_name=\"搜查结果\")\n",
    "    return (数据)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3.86 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import time\n",
    "from random import random\n",
    "time.sleep(3+4*random())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>edu</th>\n",
       "      <th>经验</th>\n",
       "      <th>薪水</th>\n",
       "      <th>时间</th>\n",
       "      <th>职称</th>\n",
       "      <th>公司地点</th>\n",
       "      <th>公司名称</th>\n",
       "      <th>链结</th>\n",
       "      <th>公司URL</th>\n",
       "      <th>curPage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>序</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>12-18k·12薪</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>商务拓展</td>\n",
       "      <td></td>\n",
       "      <td>北京德普罗尔科技有限公司</td>\n",
       "      <td>https://www.liepin.com/job/1927361265.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9583415/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>大专及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>面议</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>店长（上海国金，小众香水品牌）</td>\n",
       "      <td>北京</td>\n",
       "      <td>北京恒城实业</td>\n",
       "      <td>https://www.liepin.com/job/1927361185.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1837423/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>8-10k·12薪</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>系统管理员</td>\n",
       "      <td>无锡</td>\n",
       "      <td>无锡八佰伴</td>\n",
       "      <td>https://www.liepin.com/job/1927361121.shtml</td>\n",
       "      <td>https://www.liepin.com/company/1200071/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>经验不限</td>\n",
       "      <td>8-12k·12薪</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>KOL运营</td>\n",
       "      <td></td>\n",
       "      <td>北京和风清穆影视文化传播有限公司</td>\n",
       "      <td>https://www.liepin.com/job/1927360683.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9546719/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>大专及以上</td>\n",
       "      <td>1年以上</td>\n",
       "      <td>4-5k·12薪</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>维修技工（水电工）</td>\n",
       "      <td>云浮</td>\n",
       "      <td>翔顺集团</td>\n",
       "      <td>https://www.liepin.com/job/1927360581.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10139521/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>35</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>5年以上</td>\n",
       "      <td>9-10k·12薪</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>营销部长</td>\n",
       "      <td>湘潭</td>\n",
       "      <td>湖南新向维包装有限公司</td>\n",
       "      <td>https://www.liepin.com/job/1925688269.shtml</td>\n",
       "      <td>https://www.liepin.com/company/9104480/</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>36</td>\n",
       "      <td>硕士及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>15-30k·12薪</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>研究员</td>\n",
       "      <td></td>\n",
       "      <td>深圳市世联土地房地产评估有限公司</td>\n",
       "      <td>https://www.liepin.com/job/1925663205.shtml</td>\n",
       "      <td>https://www.liepin.com/company/849842/</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>37</td>\n",
       "      <td>本科及以上</td>\n",
       "      <td>7年以上</td>\n",
       "      <td>面议</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>(Sr.) Sales Account Manager</td>\n",
       "      <td>上海-浦东新区</td>\n",
       "      <td>MATLAB</td>\n",
       "      <td>https://www.liepin.com/job/1925634897.shtml</td>\n",
       "      <td>https://www.liepin.com/company/8185522/</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>38</td>\n",
       "      <td>学历不限</td>\n",
       "      <td>经验不限</td>\n",
       "      <td>4-5k·12薪</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>行政前台</td>\n",
       "      <td>杭州-潮鸣</td>\n",
       "      <td>浙江万邦浆纸集团有限公司</td>\n",
       "      <td>https://www.liepin.com/job/1925619251.shtml</td>\n",
       "      <td>https://www.liepin.com/company/8064651/</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>39</td>\n",
       "      <td>大专及以上</td>\n",
       "      <td>3年以上</td>\n",
       "      <td>面议</td>\n",
       "      <td>2020年04月10日</td>\n",
       "      <td>UI/UX设计师</td>\n",
       "      <td>上海</td>\n",
       "      <td>上海藤萝建筑咨询有限公司</td>\n",
       "      <td>https://www.liepin.com/job/1925616223.shtml</td>\n",
       "      <td>https://www.liepin.com/company/10250029/</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index    edu    经验          薪水           时间                           职称  \\\n",
       "序                                                                               \n",
       "0        0  本科及以上  5年以上  12-18k·12薪  2020年04月10日                         商务拓展   \n",
       "1        1  大专及以上  5年以上          面议  2020年04月10日              店长（上海国金，小众香水品牌）   \n",
       "2        2  本科及以上  3年以上   8-10k·12薪  2020年04月10日                        系统管理员   \n",
       "3        3   学历不限  经验不限   8-12k·12薪  2020年04月10日                        KOL运营   \n",
       "4        4  大专及以上  1年以上    4-5k·12薪  2020年04月10日                    维修技工（水电工）   \n",
       "..     ...    ...   ...         ...          ...                          ...   \n",
       "395     35  本科及以上  5年以上   9-10k·12薪  2020年04月10日                         营销部长   \n",
       "396     36  硕士及以上  3年以上  15-30k·12薪  2020年04月10日                          研究员   \n",
       "397     37  本科及以上  7年以上          面议  2020年04月10日  (Sr.) Sales Account Manager   \n",
       "398     38   学历不限  经验不限    4-5k·12薪  2020年04月10日                         行政前台   \n",
       "399     39  大专及以上  3年以上          面议  2020年04月10日                     UI/UX设计师   \n",
       "\n",
       "        公司地点              公司名称                                           链结  \\\n",
       "序                                                                             \n",
       "0                 北京德普罗尔科技有限公司  https://www.liepin.com/job/1927361265.shtml   \n",
       "1         北京            北京恒城实业  https://www.liepin.com/job/1927361185.shtml   \n",
       "2         无锡             无锡八佰伴  https://www.liepin.com/job/1927361121.shtml   \n",
       "3             北京和风清穆影视文化传播有限公司  https://www.liepin.com/job/1927360683.shtml   \n",
       "4         云浮              翔顺集团  https://www.liepin.com/job/1927360581.shtml   \n",
       "..       ...               ...                                          ...   \n",
       "395       湘潭       湖南新向维包装有限公司  https://www.liepin.com/job/1925688269.shtml   \n",
       "396           深圳市世联土地房地产评估有限公司  https://www.liepin.com/job/1925663205.shtml   \n",
       "397  上海-浦东新区            MATLAB  https://www.liepin.com/job/1925634897.shtml   \n",
       "398    杭州-潮鸣      浙江万邦浆纸集团有限公司  https://www.liepin.com/job/1925619251.shtml   \n",
       "399       上海      上海藤萝建筑咨询有限公司  https://www.liepin.com/job/1925616223.shtml   \n",
       "\n",
       "                                        公司URL  curPage  \n",
       "序                                                       \n",
       "0     https://www.liepin.com/company/9583415/        0  \n",
       "1     https://www.liepin.com/company/1837423/        0  \n",
       "2     https://www.liepin.com/company/1200071/        0  \n",
       "3     https://www.liepin.com/company/9546719/        0  \n",
       "4    https://www.liepin.com/company/10139521/        0  \n",
       "..                                        ...      ...  \n",
       "395   https://www.liepin.com/company/9104480/        9  \n",
       "396    https://www.liepin.com/company/849842/        9  \n",
       "397   https://www.liepin.com/company/8185522/        9  \n",
       "398   https://www.liepin.com/company/8064651/        9  \n",
       "399  https://www.liepin.com/company/10250029/        9  \n",
       "\n",
       "[400 rows x 11 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "\n",
    "list_df = list()\n",
    "for k,v in 参数_keyword_用户体验_curPage.items():\n",
    "    payload = v\n",
    "    df = requests_liepin( url, params = payload)\n",
    "    time.sleep(3+4*random())  #放慢脚步 3-7秒, 平均约5秒\n",
    "    df = df.assign (curPage = k)  # 区分  curPage\n",
    "    list_df.append(df)\n",
    "\n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "# 上周C-4   输出\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week04_liepin_翻页.xlsx\",\\\n",
    "                sheet_name=\"用户体验\")\n",
    "df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "用户体验 10\n",
      "UX 10\n"
     ]
    }
   ],
   "source": [
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "xpath_翻页a = '//div[@class=\"pagerbar\"]/a[starts-with(@href,\"/zhaopin\")]'\n",
    "\n",
    "keywords = ['用户体验','UX']\n",
    "list_df = list()\n",
    "\n",
    "## 第一页试探有多长的页面\n",
    "for key in keywords:\n",
    "    payload = 参数模板生成(keyword=[key], curPage=['0'])\n",
    "    df = requests_liepin( url, params = payload)\n",
    "    href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "    df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "    df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "    df_qs = df_qs.assign (curPage_int=df_qs.curPage.astype(int)) # 变成整数\n",
    "    长度 = df_qs.curPage_int.max()+1\n",
    "    参数_keyword_X_curPage = { \n",
    "        i:参数模板生成(curPage = [i], \\\n",
    "                      keyword = [key]) \\\n",
    "        for i in range(0,长度)\\\n",
    "        }\n",
    "    #print (参数_keyword_X_curPage)\n",
    "    print (key,长度)\n",
    "    \n",
    "    for k,v in 参数_keyword_X_curPage.items():\n",
    "        payload = v\n",
    "        df = requests_liepin( url, params = payload)\n",
    "        time.sleep(3+4*random())  #放慢脚步 3-7秒, 平均约5秒\n",
    "        df = df.assign (keyword = key)  # 区分  keyword    \n",
    "        df = df.assign (curPage = k)  # 区分  curPage    \n",
    "        list_df.append(df)\n",
    "        \n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week04_liepin_翻页1.xlsx\",\\\n",
    "                sheet_name=\"_\".join(keywords))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "用户体验 10\n",
      "UX 10\n",
      "产品需求 10\n",
      "PRD 10\n"
     ]
    }
   ],
   "source": [
    "url = \"https://www.liepin.com/zhaopin/\"\n",
    "xpath_翻页a = '//div[@class=\"pagerbar\"]/a[starts-with(@href,\"/zhaopin\")]'\n",
    "\n",
    "keywords = ['用户体验','UX','产品需求','PRD']\n",
    "list_df = list()\n",
    "\n",
    "## 第一页试探有多长的页面\n",
    "for key in keywords:\n",
    "    payload = 参数模板生成(keyword=[key], curPage=['0'])\n",
    "    df = requests_liepin( url, params = payload)\n",
    "    href_列表 = [x.xpath('//@href')[0] for x in r.html.xpath(xpath_翻页a)]\n",
    "    df = pd.DataFrame([ urlparse(x) for x in href_列表])\n",
    "    df_qs = pd.DataFrame([{k:v[0] for k,v in parse_qs(x).items()} for x in df['query'] ])\n",
    "    df_qs = df_qs.assign (curPage_int=df_qs.curPage.astype(int)) # 变成整数\n",
    "    长度 = df_qs.curPage_int.max()+1\n",
    "    参数_keyword_X_curPage = { \n",
    "        i:参数模板生成(curPage = [i], \\\n",
    "                      keyword = [key]) \\\n",
    "        for i in range(0,长度)\\\n",
    "        }\n",
    "    #print (参数_keyword_X_curPage)\n",
    "    print (key,长度)\n",
    "    \n",
    "    for k,v in 参数_keyword_X_curPage.items():\n",
    "        payload = v\n",
    "        df = requests_liepin( url, params = payload)\n",
    "        time.sleep(3+4*random())  #放慢脚步 3-7秒, 平均约5秒\n",
    "        ## 备份\n",
    "        df.to_csv(\"20春_Web数据挖掘_week04_liepin_{key}_{k}.tsv\"\\\n",
    "                  .format(key=key, k=k), sep=\"\\t\", encoding=\"utf8\")\n",
    "        \n",
    "        df = df.assign (keyword = key)  # 区分  keyword    \n",
    "        df = df.assign (curPage = k)  # 区分  curPage    \n",
    "        list_df.append(df)\n",
    "        \n",
    "df_all = pd.concat(list_df).reset_index()\n",
    "df_all.index.name = '序'\n",
    "\n",
    "df_all.to_excel(\"20春_Web数据挖掘_week04_liepin_翻页_4.xlsx\",\\\n",
    "                sheet_name=\"_\".join(keywords))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
